'''
Created on Sep 20, 2012

@author: kingsfield
'''
import csv
import re
from collections import defaultdict
from util import Util
import train_word_correct

train_file = '/home/kingsfield/Desktop/BestBuyPc/old_train.csv'
test_file = '/home/kingsfield/Desktop/BestBuyPc/old_test.csv'
product_file = '/home/kingsfield/Desktop/BestBuyPc/small_product_data.xml'
normal_file = 'big.txt'

def readfile(f):
    #with open(file) as infile:
    infile = open(f)
    reader = csv.reader(infile, delimiter=",")
    reader.next() # burn the header
    return reader

def words(text): return re.findall('[a-z]+', text.lower())

def get_count(features):
    model = defaultdict(lambda: 1)
    for f in features:
        model[f] += 1
    return model

def get_normal_words():
    with open(normal_file) as fr:
        text = fr.read()
        return get_count(words(text))
    
def get_products_words():
    with open(product_file) as fr:
        text = fr.read()
        return get_count(words(text))

def get_hot_word(thr):
    """get the item word count"""
    reader = readfile(train_file)
    hot_word = defaultdict(int)
    for (__user, __sku, __category, raw_query, ___click_time, ___query_time) in reader:
        words = Util.wordutil.get_words(raw_query)
        for w in words:
            hot_word[w] += 1
    hot_word = sorted(hot_word.items(), key=lambda x: x[0], reverse=True)
    hot_word = dict([w for w in hot_word if w[1] >= thr])
    return hot_word

if __name__ == '__main__':
#    p_words = get_products_words()
#    n_words = get_normal_words()
#    hot_word = get_hot_word(1)
#    print 'hot_word=%d' % len(hot_word)
#    filtered_word = [[w, hot_word[w]] for w in hot_word if w in p_words if w in n_words]
#    print 'filtered_word=%d' % len(filtered_word)
    #print 'filtered_word=%s' % filtered_word
    #print train_word_correct.filter_correct_word({'soccer':1})
    reader = readfile(test_file) 
    #for (__user, sku, __category, raw_query, ___click_time, ___query_time) in reader:
    for (__user, __category, raw_query, ___click_time, ___query_time) in reader:
        words = Util.wordutil.get_correct_words(raw_query)
        if len(words) == 0 or words == ' ':
            print '%s %s' % (1, raw_query)
    print 'done'
